QuantConnect Logo


Get Backtest Orders¶

Get the backtest orders that you saved into the Object Store when you ran the benchmark and candidate algorithms in main.py.

In [ ]:
for kvp in qb.object_store:
    key = kvp.key
    value = kvp.value
    print(key, value)
In [ ]:
qb = QuantBook()
def get_order_fills(key):
    return pd.read_csv(
        qb.object_store.get_file_path(f"{key}_order_fills"), 
        index_col=0, parse_dates=True
    )
benchmark_orders = get_order_fills("benchmark")
candidate_orders = get_order_fills("candidate")

Calculate Summary Statistics¶

The following code block calculates some statistics and performs some sanity tests on the orders data.

In [6]:
same_index = candidate_orders.index == benchmark_orders.index
matching_fill_times = candidate_orders[same_index].index
different_fill_times_for_candidate = candidate_orders[~same_index].index
different_fill_times_for_benchmark = benchmark_orders[~same_index].index

print(f"Number of trades: {len(benchmark_orders)}")
print("Number of trades with the same fill times:", len(matching_fill_times))
print(
    "Number of trades with different fill times:", 
    len(different_fill_times_for_candidate)
)
costs_saved = sum(
    benchmark_orders['cost'].values - candidate_orders['cost'].values
)
print("Costs saved by delaying orders:", f"${round(costs_saved, 2)}")

# For all pairs of orders between the two algorithms, ensure the 
# quantities match.
candidate_quantities = candidate_orders['quantity'].values
benchmark_quantities = benchmark_orders['quantity'].values
if not all(candidate_quantities == benchmark_quantities):
    raise Exception('Error: The algorithms traded different quantities')

# For the orders that fill at the same time for both algorithms, ensure 
# the fill price and costs also match.
candidate_info = candidate_orders.loc[matching_fill_times].drop('tag', axis=1)
benchmark_info = benchmark_orders.loc[matching_fill_times].drop('tag', axis=1)
if not all(candidate_info == benchmark_info):
    raise Exception(
        'Error: Fill prices and costs of identical orders do not match'
    )
Number of trades: 366
Number of trades with the same fill times: 135
Number of trades with different fill times: 231
Costs saved by delaying orders: $6918.3

Analysis of Delayed Orders That Filled Before the Time Limit¶

Let's take a look at just the exit orders that were delayed but didn't hit the time limit and see how their costs compare to the costs of the benchmark orders.

In [7]:
from collections import Counter
import plotly.graph_objects as go

# Calculate number of orders that the algorithm delayed (and didn't hit 
# the time limit).
index_numbers_of_time_limited_orders = []
index_numbers_of_delayed_orders = []
for i in range(len(benchmark_orders.index)):
    # Skip orders that filled at the same time.
    if benchmark_orders.index[i] == candidate_orders.index[i]:
        continue
    # Skip orders that hit the time limit.
    if "Hit time limit" in candidate_orders.iloc[i]['tag']:
        index_numbers_of_time_limited_orders.append(i)
        continue
    index_numbers_of_delayed_orders.append(i)

def display_order_results(title, index_numbers):
    cost_deltas = (
        benchmark_orders.iloc[index_numbers]['cost'].values 
        - candidate_orders.iloc[index_numbers]['cost'].values
    )
    candidate_dollar_volume = (
        abs(candidate_orders.iloc[index_numbers]['quantity']) 
        * candidate_orders.iloc[index_numbers]['fill_price']
    ).values
    cost_deltas_per_dollar_volume = cost_deltas / candidate_dollar_volume

    fig = go.Figure(data=[go.Histogram(x=cost_deltas_per_dollar_volume, nbinsx=100)])
    fig.update_layout(
        title="Distribution of Costs Saved<br><sup>Costs saved for "
            + f"Delayed Orders that {title}</sup>",
        xaxis_title="Costs Saved Per Dollar Volume (>0 => Candidate Saved Money)",
        yaxis_title="Count"
    )
    fig.show()

    print("Number of orders:", len(index_numbers))
    pct_of_cheaper_orders = (
        len(cost_deltas[cost_deltas > 0]) 
        / len(candidate_orders) * 100
    )
    print(
        "Number of orders that lowered costs:", 
        len(cost_deltas[cost_deltas > 0]),
        f"({round(pct_of_cheaper_orders, 2)}% of all orders)"
    )
    pct_of_no_change = (
        len(cost_deltas[cost_deltas == 0]) 
        / len(candidate_orders) * 100
    )
    print(
        "Number of orders that didn't change costs:", 
        len(cost_deltas[cost_deltas == 0]),
        f"({round(pct_of_no_change, 2)}% of all orders)"
    )
    pct_of_raised_orders = (
        len(cost_deltas[cost_deltas < 0]) 
        / len(candidate_orders) * 100
    )
    print(
        "Number of orders that raised costs:",
        len(cost_deltas[cost_deltas < 0]),
        f"({round(pct_of_raised_orders, 2)}% of all orders)"
    )
    print(f"Total costs saved: ${round(sum(cost_deltas), 2)}")

display_order_results(
    "Filled Before the Time Limit", index_numbers_of_delayed_orders
)

minutes_delayed = (
    candidate_orders.iloc[index_numbers_of_delayed_orders].index
    - benchmark_orders.iloc[index_numbers_of_delayed_orders].index
)
num_orders_by_delay_duration = pd.Series(
    Counter([x.total_seconds() / 60 for x in minutes_delayed])
).sort_index()

fig = go.Figure(
    go.Scatter(
        x=num_orders_by_delay_duration.index, 
        y=num_orders_by_delay_duration.values, mode='lines'
    )
)
fig.update_layout(
    title="Number of Orders Filled Per Delay Duration<br><sup>Most order fill "
        + "within 90 minutes</sup>",
    xaxis_title="Minutes Delayed",
    yaxis_title="Number of Orders Filled"
)
fig.show()
Number of orders: 219
Number of orders that lowered costs: 218 (59.56% of all orders)
Number of orders that didn't change costs: 0 (0.0% of all orders)
Number of orders that raised costs: 1 (0.27% of all orders)
Total costs saved: $6852.0

Analysis of Delayed Orders That Hit the Time Limit¶

Now let's take a look at just the exit orders that hit the time limit and how their costs compare to the costs of the benchmark orders.

In [8]:
display_order_results(
    "Hit the Time Limit", index_numbers_of_time_limited_orders
)
Number of orders: 12
Number of orders that lowered costs: 4 (1.09% of all orders)
Number of orders that didn't change costs: 4 (1.09% of all orders)
Number of orders that raised costs: 4 (1.09% of all orders)
Total costs saved: $66.3